#pragma once

#include <linux/bug.h>
#include <linux/types.h>
#include <riscv/fence.h>
#include <linux/bits.h>

#define __arch_xchg_masked(sc_sfx, swap_sfx, prepend, sc_append, \
                           swap_append, r, p, n)                 \
    ({                                                           \
        __asm__ __volatile__(                                    \
            prepend                                              \
            " amoswap" swap_sfx " %0, %z2, %1\n" swap_append     \
            : "=&r"(r), "+A"(*(p))                               \
            : "rJ"(n)                                            \
            : "memory");                                         \
    })

/*
 * Atomic compare and exchange.  Compare OLD with MEM, if identical,
 * store NEW in MEM.  Return the initial value in MEM.  Success is
 * indicated by comparing RETURN with OLD.
 */
#define __arch_cmpxchg_masked(sc_sfx, cas_sfx,                         \
                              sc_prepend, sc_append,                   \
                              cas_prepend, cas_append,                 \
                              r, p, o, n)                              \
    ({                                                                 \
        u32 *__ptr32b = (u32 *)((ulong)(p) & ~0x3);                    \
        ulong __s = ((ulong)(p) & (0x4 - sizeof(*p))) * BITS_PER_BYTE; \
        ulong __mask = GENMASK(((sizeof(*p)) * BITS_PER_BYTE) - 1, 0)  \
                       << __s;                                         \
        ulong __newx = (ulong)(n) << __s;                              \
        ulong __oldx = (ulong)(o) << __s;                              \
        ulong __retx;                                                  \
        ulong __rc;                                                    \
                                                                       \
        __asm__ __volatile__(                                          \
            sc_prepend                                                 \
            "0:	lr.w %0, %2\n"                                         \
            "	and  %1, %0, %z5\n"                                    \
            "	bne  %1, %z3, 1f\n"                                    \
            "	and  %1, %0, %z6\n"                                    \
            "	or   %1, %1, %z4\n"                                    \
            "	sc.w" sc_sfx " %1, %1, %2\n"                           \
            "	bnez %1, 0b\n" sc_append                               \
            "1:\n"                                                     \
            : "=&r"(__retx), "=&r"(__rc), "+A"(*(__ptr32b))            \
            : "rJ"((long)__oldx), "rJ"(__newx),                        \
              "rJ"(__mask), "rJ"(~__mask)                              \
            : "memory");                                               \
                                                                       \
        r = (__typeof__(*(p)))((__retx & __mask) >> __s);              \
    })

#define __arch_cmpxchg(lr_sfx, sc_sfx, cas_sfx, \
                       sc_prepend, sc_append,   \
                       cas_prepend, cas_append, \
                       r, p, co, o, n)          \
    ({                                          \
        register unsigned int __rc;             \
                                                \
        __asm__ __volatile__(                   \
            sc_prepend                          \
            "0:	lr" lr_sfx " %0, %2\n"          \
            "	bne  %0, %z3, 1f\n"             \
            "	sc" sc_sfx " %1, %z4, %2\n"     \
            "	bnez %1, 0b\n" sc_append        \
            "1:\n"                              \
            : "=&r"(r), "=&r"(__rc), "+A"(*(p)) \
            : "rJ"(co o), "rJ"(n)               \
            : "memory");                        \
    })

#define _arch_cmpxchg(ptr, old, new, sc_sfx, cas_sfx,           \
                      sc_prepend, sc_append,                    \
                      cas_prepend, cas_append)                  \
    ({                                                          \
        __typeof__(ptr) __ptr = (ptr);                          \
        __typeof__(*(__ptr)) __old = (old);                     \
        __typeof__(*(__ptr)) __new = (new);                     \
        __typeof__(*(__ptr)) __ret;                             \
                                                                \
        switch (sizeof(*__ptr))                                 \
        {                                                       \
        case 1:                                                 \
            __arch_cmpxchg_masked(sc_sfx, ".b" cas_sfx,         \
                                  sc_prepend, sc_append,        \
                                  cas_prepend, cas_append,      \
                                  __ret, __ptr, __old, __new);  \
            break;                                              \
        case 2:                                                 \
            __arch_cmpxchg_masked(sc_sfx, ".h" cas_sfx,         \
                                  sc_prepend, sc_append,        \
                                  cas_prepend, cas_append,      \
                                  __ret, __ptr, __old, __new);  \
            break;                                              \
        case 4:                                                 \
            __arch_cmpxchg(".w", ".w" sc_sfx, ".w" cas_sfx,     \
                           sc_prepend, sc_append,               \
                           cas_prepend, cas_append,             \
                           __ret, __ptr, (long), __old, __new); \
            break;                                              \
        case 8:                                                 \
            __arch_cmpxchg(".d", ".d" sc_sfx, ".d" cas_sfx,     \
                           sc_prepend, sc_append,               \
                           cas_prepend, cas_append,             \
                           __ret, __ptr, /**/, __old, __new);   \
            break;                                              \
        default:                                                \
            BUILD_BUG();                                        \
        }                                                       \
        (__typeof__(*(__ptr)))__ret;                            \
    })

/*
 * These macros are here to improve the readability of the arch_cmpxchg_XXX()
 * macros.
 */
#define SC_SFX(x) x
#define CAS_SFX(x) x
#define SC_PREPEND(x) x
#define SC_APPEND(x) x
#define CAS_PREPEND(x) x
#define CAS_APPEND(x) x

#define arch_cmpxchg_relaxed(ptr, o, n)          \
    _arch_cmpxchg((ptr), (o), (n),               \
                  SC_SFX(""), CAS_SFX(""),       \
                  SC_PREPEND(""), SC_APPEND(""), \
                  CAS_PREPEND(""), CAS_APPEND(""))

#define arch_cmpxchg_acquire(ptr, o, n)                             \
    _arch_cmpxchg((ptr), (o), (n),                                  \
                  SC_SFX(""), CAS_SFX(""),                          \
                  SC_PREPEND(""), SC_APPEND(RISCV_ACQUIRE_BARRIER), \
                  CAS_PREPEND(""), CAS_APPEND(RISCV_ACQUIRE_BARRIER))

#define arch_cmpxchg_release(ptr, o, n)                             \
    _arch_cmpxchg((ptr), (o), (n),                                  \
                  SC_SFX(""), CAS_SFX(""),                          \
                  SC_PREPEND(RISCV_RELEASE_BARRIER), SC_APPEND(""), \
                  CAS_PREPEND(RISCV_RELEASE_BARRIER), CAS_APPEND(""))

#define arch_cmpxchg(ptr, o, n)                                  \
    _arch_cmpxchg((ptr), (o), (n),                               \
                  SC_SFX(".rl"), CAS_SFX(".aqrl"),               \
                  SC_PREPEND(""), SC_APPEND(RISCV_FULL_BARRIER), \
                  CAS_PREPEND(""), CAS_APPEND(""))

#define arch_cmpxchg_local(ptr, o, n) \
    arch_cmpxchg_relaxed((ptr), (o), (n))

#define arch_cmpxchg64(ptr, o, n)          \
    ({                                     \
        BUILD_BUG_ON(sizeof(*(ptr)) != 8); \
        arch_cmpxchg((ptr), (o), (n));     \
    })

#define arch_cmpxchg64_local(ptr, o, n)        \
    ({                                         \
        BUILD_BUG_ON(sizeof(*(ptr)) != 8);     \
        arch_cmpxchg_relaxed((ptr), (o), (n)); \
    })

#define arch_cmpxchg64_relaxed(ptr, o, n)      \
    ({                                         \
        BUILD_BUG_ON(sizeof(*(ptr)) != 8);     \
        arch_cmpxchg_relaxed((ptr), (o), (n)); \
    })

#define arch_cmpxchg64_acquire(ptr, o, n)      \
    ({                                         \
        BUILD_BUG_ON(sizeof(*(ptr)) != 8);     \
        arch_cmpxchg_acquire((ptr), (o), (n)); \
    })

#define arch_cmpxchg64_release(ptr, o, n)      \
    ({                                         \
        BUILD_BUG_ON(sizeof(*(ptr)) != 8);     \
        arch_cmpxchg_release((ptr), (o), (n)); \
    })

#define __arch_xchg(sfx, prepend, append, r, p, n)  \
    ({                                              \
        __asm__ __volatile__(                       \
            prepend                                 \
            "	amoswap" sfx " %0, %2, %1\n" append \
            : "=r"(r), "+A"(*(p))                   \
            : "r"(n)                                \
            : "memory");                            \
    })

#define _arch_xchg(ptr, new, sc_sfx, swap_sfx, prepend,         \
                   sc_append, swap_append)                      \
    ({                                                          \
        __typeof__(ptr) __ptr = (ptr);                          \
        __typeof__(*(__ptr)) __new = (new);                     \
        __typeof__(*(__ptr)) __ret;                             \
                                                                \
        switch (sizeof(*__ptr))                                 \
        {                                                       \
        case 1:                                                 \
            __arch_xchg_masked(sc_sfx, ".b" swap_sfx,           \
                               prepend, sc_append, swap_append, \
                               __ret, __ptr, __new);            \
            break;                                              \
        case 2:                                                 \
            __arch_xchg_masked(sc_sfx, ".h" swap_sfx,           \
                               prepend, sc_append, swap_append, \
                               __ret, __ptr, __new);            \
            break;                                              \
        case 4:                                                 \
            __arch_xchg(".w" swap_sfx, prepend, swap_append,    \
                        __ret, __ptr, __new);                   \
            break;                                              \
        case 8:                                                 \
            __arch_xchg(".d" swap_sfx, prepend, swap_append,    \
                        __ret, __ptr, __new);                   \
            break;                                              \
        default:                                                \
            BUILD_BUG();                                        \
        }                                                       \
        (__typeof__(*(__ptr)))__ret;                            \
    })

#define arch_xchg_acquire(ptr, x)  \
    _arch_xchg(ptr, x, "", "", "", \
               RISCV_ACQUIRE_BARRIER, RISCV_ACQUIRE_BARRIER)
